# coding: utf-8
import urllib
import urllib2
import cookielib
from HTMLParser import HTMLParser
import re
import xlrd
import xlwt
import string
from urllib2 import Request, urlopen, URLError, HTTPError
import requests

#=====================================================

def getStrvalue(tar,strname,spstr="&nbsp;"):
    if tar.find(strname)>=0:
        ress=tar.split(strname)
        print "可以切割"+strname+"   切割后长度："+str(len(ress))
        if len(ress)>1:
            ress = ress[1].split(spstr)
            ress = ress[1].split("<")
            return ress[0]
        else:
            print "切割出错，字符串没有："+strname
    return ""


#======================================================
class myHtmlParser(HTMLParser):
    irstart=0
    irend=0
    result=[]
    data=[]
    isfinids=0
    iscleasspace=0
    def __init__(self):
        HTMLParser.__init__(self)
        self.flag=None
    def handle_starttag(self,tag,attrs):
        if self.irstart==1:
            if tag=='a':
                self.flag='a'
                for href,link in attrs:
                    if href=='href':
                        if self.isfinids==1:
                            if link!='/newsAction.do?method=viewNews&classId=020019980000000472&newsId=020010040000016998' and link!='/newsAction.do?method=viewNews&classId=020010350000000877&newsId=020010040000108745':
                                if link.find('/newsAction.do?method=viewNews&classId')>=0:
                                    self.result.append(link)
                        else:
                            self.result.append(link)





    def handle_data(self,data):
        if self.irend==1:
            sdata = data.strip()
            if sdata!='':
                if self.iscleasspace==1:
                    sdata=sdata.replace("\r\n","")
                    sdata=sdata.replace(" ","")
                    sdata=sdata.replace(' ','')
                self.data.append(sdata)
                #print 'data:'+sdata

#======================================================
#http://www.gdlr.gov.cn/newsAction.do?method=viewNews&classId=020010350000000680&newsId=020010040000205418
fdcinfolist=[]

i=0
dikuaibianma = "" #地块编码
dikuaiweizhi = "" #地块位置
tudiyongtu = "" #土地用途
area=""#土地面积（公顷）
selldate=""#出让年限
sellPrice=""#成交价
unit =""#受让单位src
des = ""#备注
ptime=""#发布时间
#获取所有网址


findpage = 1
while findpage<=24:
    print str(findpage)+"获取所有id"
    cookie = cookielib.CookieJar()
    opener=urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie))
    url='http://www.gdlr.gov.cn/newsAction.do?method=searchNews'
    tilename = u'高明区'
    tilename = tilename.encode("gb2312")
    values={
        'classId':'null',
        'textGeneralType':'020010350000000679',
        'newsCatalogID':'020010350000000679',
        'isquery':'',
        'type':'',
        'type_name':'',
        'oldTitle':tilename,'oldTitle':'','titleName':tilename,'linkUrl':'',
        '020010350000000679':'','strTime':'','endTime':'','orderSign':'0','curPageNo':'1',
        'totalCnts':'236','totalPages':'24','cntPerPage':'10','SplitFlag':'1','orderBy':'null','descOrAsc':'desc','gotoPage':findpage}
    data = urllib.urlencode(values)
    req = urllib2.Request(url,data)
    req.add_header('Host','www.gdlr.gov.cn')
    req.add_header('Referer','http://www.gdlr.gov.cn/newsAction.do?method=searchNews&textGeneralType=020010350000000679')
    req.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.3; WOW64; rv:39.0) Gecko/20100101 Firefox/39.0')
    response = opener.open(req)
    the_page=response.read()
    pstrl = unicode(the_page,"gb2312",'ignore').encode("utf8")
    response.close()
    m= myHtmlParser()
    m.irstart=1
    m.isfinids=1
    m.feed(pstrl)
    m.close()
    result = m.result
    for itemstr in result:
        fdcinfolist.append(itemstr)
    findpage+=1

print "删除重复前:"+str(len(fdcinfolist))
fdcinfolist =  list(set(fdcinfolist))
print "删除重复后:"+str(len(fdcinfolist))

allDic=[]
#创建表格
f= xlwt.Workbook()#创建工作簿
sheetname = u"成交公示"
sheet = f.add_sheet(sheetname,cell_overwrite_ok=True)
#行头
print "创建表格"

readPos = -1
pppppp=1
for fdcid in fdcinfolist:
    readPos+=1
    print  "=获取id:"+fdcid
    url = 'http://www.gdlr.gov.cn'+fdcid
    print '浏览网页:'+url
    bskk =0
    botiaozhuan = 0
    try:
        headers = requests.head(url).headers

        for kkk in headers:
            if kkk=='location':
                url= headers['location']
                botiaozhuan=1
    except :
        botiaozhuan=0


    request = urllib2.Request(url)
    print '跳转网页:'+url
    if botiaozhuan==1:
        request.add_header('Host','www.landgd.com')
    else:
        request.add_header('Host',' www.gdlr.gov.cn')
    request.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.3; WOW64; rv:39.0) Gecko/20100101 Firefox/39.0')

    try:
        response = urllib2.urlopen(request)
    except :
        continue

    print str(readPos)+"读取网址"+url
    strl = response.read()
    response.close()
    pstrl = unicode(strl,"gb2312",'ignore').encode("utf8")
    strl = pstrl
    strl=strl.replace("\r\n","")
    strl=strl.replace(" ","")
    p=re.compile('\s+')
    strl=re.sub(p,'',strl)
    s="====================="
    lencount =2
    results = strl.split("地块编号")
    #==================================================================

    finddx = strl.find("项目名称")
    print "================================="
    print strl
    #获取发布时间
    sltimes = strl.split("发布时间：")
    ptime=""
    if len(sltimes)>1:
        nstr = sltimes[1]
        sltimes = nstr.split("<")
        ptime=sltimes[0]
    print "================================="

    if finddx<0:

        print len(results)
        ldj ="找到了"
        if(len(results) ==lencount):
            vki = 1
            while vki<len(results):
                clk = results[vki]
                vki+=1
                clk = clk.replace("</td>","")
                finds = clk.split("<td>")
                ind = 0
                for vk in finds:
                    print str(ind)+s
                    print vk
                    ind+=1

                nki=0
                lop=0
                while lop<len(finds):
                    ntr = finds[lop]
                    lop+=1
                    finddx = ntr.find("工业用地")
                    if finddx>=0:
                        nki=1
                        break

                if nki==1:
                    continue
                if finds[1]=="地块位置" and finds[2]=="土地面积(公顷)":
                    dikuaibianma=finds[7]
                    dikuaiweizhi=finds[8]
                    tudiyongtu=finds[10]
                    area=finds[9]
                    selldate=finds[11]
                    sellPrice=finds[12]
                    unit=finds[13]
                    des=""
                else:
                    ustr = finds[1].split("<")
                    dikuaibianma = ustr[0] #获取地块编码
                    #===========================
                    ustr=  finds[2].split("<") #获取地块位置
                    dikuaiweizhi = ustr[0]
                    #===========================
                    ustr=  finds[3].split("<") #获取土地用途
                    tudiyongtu = ustr[0]
                    #===========================
                    ustr=  finds[5].split("<") #获取土地面积
                    area = ustr[0]
                    #===========================
                    ustr=  finds[7].split("<") #获取出让年限
                    selldate = ustr[0]
                    #===========================
                    ustr=  finds[9].split("<") #获取成交价
                    sellPrice= ustr[0]
                    #===========================
                    ustr=  finds[10].split(">") #获取受让单位
                    unit = ustr[1].split("<")[0]
                    #===========================
                    ntr = finds[11]
                    fdx = ntr.find("土地使用条件")
                    des=""
                    if fdx>=0:
                        ntr = ntr.replace("</tr","")
                        ustr= ntr.split(">") #获取备注
                        des=ustr[1]
                        des="土地使用条件:"+des
                    else:
                        fdx = ntr.find("备注")
                        if fdx<0:
                            if len(finds)>12:
                                ntr=finds[12]
                                ustr= ntr.split("nbsp;") #获取备注
                                des=ustr[1].split("<")[0]
                            else:
                                des=""



                objDic=[]
                objDic.append(ptime)
                objDic.append(dikuaibianma)
                objDic.append(dikuaiweizhi)
                objDic.append(tudiyongtu)
                objDic.append(area)
                objDic.append(selldate)
                objDic.append(sellPrice)
                objDic.append(unit)
                objDic.append(des)

                allDic.append(objDic)
                print "==============================="
                print "  "
                print  "公示时间："+ptime
                print  "地块编码："+dikuaibianma
                print  "地块位置："+dikuaiweizhi
                print  "土地用途："+tudiyongtu
                print  "土地面积："+area
                print  "出让年限："+selldate
                print  "成交价："+sellPrice
                print  "受让单位："+unit
                print  "备注："+des

f= xlwt.Workbook()#创建工作簿
sheetname = "成交公示"
sheet = f.add_sheet(unicode(sheetname,'utf-8'),cell_overwrite_ok=True)
#创建表头
sheet.write(0,0,u'公示时间')
sheet.write(0,1,u'地块编码')
sheet.write(0,2,u"地块位置")
sheet.write(0,3,u"土地用途")
sheet.write(0,4,u"土地面积(公顷)")
sheet.write(0,5,u"出让年限")
sheet.write(0,6,u"成交价(万元)")
sheet.write(0,7,u"受让单位")
sheet.write(0,8,u"备注")


nrows = len(allDic)
i=0
while i<nrows:
    wdic = allDic[i]
    k=i+1
    print "新的数据，开始写入档案"
    sheet.write(k,0,unicode(wdic[0],'utf-8'))
    sheet.write(k,1,unicode(wdic[1],'utf-8'))
    sheet.write(k,2,unicode(wdic[2],'utf-8'))
    sheet.write(k,3,unicode(wdic[3],'utf-8'))
    sheet.write(k,4,unicode(wdic[4],'utf-8'))
    sheet.write(k,5,unicode(wdic[5],'utf-8'))
    sheet.write(k,6,unicode(wdic[6],'utf-8'))
    sheet.write(k,7,unicode(wdic[7],'utf-8'))
    sheet.write(k,8,unicode(wdic[8],'utf-8'))
    i+=1

f.save( u'高明土地信息（成交公示）.xls')




